{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bert\n",
    "import bert.modeling, bert.tokenization\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/husein/parsing/self-attentive-parser/src\")\n",
    "sys.path.append(\"/home/husein/parsing/self-attentive-parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_tiny_bert as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/en_bert_dev=76.79.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    config = bert.modeling.BertConfig.from_dict(parser.bert.config.to_dict())\n",
    "    model = bert.modeling.BertModel(config=config, is_training=False,\n",
    "        input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)\n",
    "\n",
    "    # Next, transfer learned weights (after fine-tuning)\n",
    "    bert_variables = [v for v in tf.get_collection('variables') if 'bert' in v.name]\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    \n",
    "    for variable in bert_variables:\n",
    "        name = variable.name.split(':')[0]\n",
    "        name = name.split('/')\n",
    "        array = variable.eval()\n",
    "        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "        # which are not required for using pretrained model\n",
    "        if any(n in [\"adam_v\", \"adam_m\"] for n in name):\n",
    "            print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "            continue\n",
    "        pytorch_var = parser\n",
    "        for m_name in name:\n",
    "            if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "                l = re.split(r'_(\\d+)', m_name)\n",
    "            else:\n",
    "                l = [m_name]\n",
    "            if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "                pytorch_var = getattr(pytorch_var, 'bias')\n",
    "            elif l[0] == 'output_weights':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'cls':\n",
    "                pytorch_var = getattr(pytorch_var, 'cls')\n",
    "            else:\n",
    "                pytorch_var = getattr(pytorch_var, l[0])\n",
    "            if len(l) >= 2:\n",
    "                try:\n",
    "                    num = int(l[1])\n",
    "                    pytorch_var = pytorch_var[num]\n",
    "                except Exception as e:\n",
    "                    print(pytorch_var)\n",
    "                    raise\n",
    "        if m_name[-11:] == '_embeddings':\n",
    "            pytorch_var = getattr(pytorch_var, 'weight')\n",
    "        elif m_name == 'kernel':\n",
    "            pytorch_var = pytorch_var.t()\n",
    "        try:\n",
    "            assert pytorch_var.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            print(e)\n",
    "            e.args += (pytorch_var.shape, array.shape)\n",
    "            raise\n",
    "            \n",
    "        variable.load(pytorch_var.detach().cpu().numpy())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:233: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-12-f98425b040db>:54: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "WARNING:tensorflow:From <ipython-input-9-209ca551d3eb>:10: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-9-209ca551d3eb>:10: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-11-dc457fb7a117>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        tokens.append(\"[CLS]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:02<00:00, 11.31it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7f95a004a978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d8a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f95a006d128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3db668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f95a00084a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3de198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f95a001ac88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3decf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f708588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3e1908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de87e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3e4390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de8b748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3e4eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de900f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3e79e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de92be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3ea6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de95828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3ed2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de99438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3edfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de9e048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3f0c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dea0ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3f3908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dea4828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3f6588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dea7438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3f8278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deaae48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3f8f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deaea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c9eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deb2c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d2b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deb3940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d4048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f95a00cea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d4cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deb7a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3ce0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958deb9940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3cebe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958debb860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d32e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958debc780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3d39b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958debd6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3810f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958debe588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3817b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dec0470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c381e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dec1390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3835c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dec22b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c383c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dec4780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c387320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f7142b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c387cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f714d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38a208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f748860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38a7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f744390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38ada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f744e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c1278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f7429b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c16d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f735470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c1b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f735f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c1f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f73afd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38c470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f72b048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38f160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f723160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c38fdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f722320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c393b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958f7194e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c396320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de476d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c396b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de4a940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c398400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de4cb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c398cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de4ecc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c39b630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de50eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c39beb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de54160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3a4ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de570f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3a7278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de5acc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3a7fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de5dc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3aa240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de5fc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3aa8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de61c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3aaeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de63cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3ae588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de64c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3aec88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de69c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b1358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de6bc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b19b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de6cc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b1fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de707b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b4710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de73320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b73c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de73a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3b7fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de779e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3ba390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de77f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bc278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7a4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c21d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7a9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7aef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7c470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7ca20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7cf98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de7e518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c2f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de80ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c52b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de85be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c9780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de08668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c39efd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de09128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3a40f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de09b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3a4518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de0c5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bc6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de0f080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bce10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de0fac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c33f320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de10550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c33f7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de10f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c351ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de13588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c351fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de13dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c353a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de16668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c353e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de16f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3561d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de18828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c356518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de190b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3568d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de19978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c356c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de1c278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c359080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de1cb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c359400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de1f2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c359780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de23978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c35c2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de2a080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c35f550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de2e630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3626d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de32cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c365860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de393c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3689e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de3da20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2fffd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de420f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c305208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de44710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c305be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc6cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c308668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddca438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c30b0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddcca90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c30bac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddd1208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c30e4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddd3860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c30eef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddd4ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3119e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddd9668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3133c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dddce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c313dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dde45c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c31b160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dde8da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c31d5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddee4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c34ba58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddf3cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c321e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddf94e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c329278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddfc320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c34d630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddfe128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3416d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddfef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c341be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de00d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c341d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de02ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c373470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958de059e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c373c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd87828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c377198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd88668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c377860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd8ae10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c379390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd8f5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c379da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd91d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c37c390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd96470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c32bcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd98c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c32e780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd9c438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c332208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd9ec18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c332c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda3438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c335710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda3a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c06d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda60b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c33fcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda66d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda6cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda72e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda78d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3493c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dda7ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddaa550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddae550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c349e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddb2080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c34d080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddb5668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c338c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddb9a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2be5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddbf828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c0d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddbfcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c39b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc2240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c3ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc2748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c3da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc2c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c65f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc3128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c67f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc3630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c69e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc3b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c6be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958ddc5048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c6dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd4a940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2c6fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd4b780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2ce320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd4d630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2cea20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd50400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d1128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd53208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d1860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd55080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d1f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd55e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d36a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd57c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d3e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd59a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d6550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd5e278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2d6cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd5fb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2da6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd64438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2f17f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd66e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2f3320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd6a780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2f3d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd6f128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2f7748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd70b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2fa278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd74400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2fac50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd79748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2fc5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd7bf98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c280710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd7f7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c283358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd850b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c283f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd06940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c285b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd0b1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2897b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd0ca58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c28c438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd112e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c28e080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd13b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c28ec88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd16e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2918d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd1e0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c299748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd22438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c29b940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd27748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c29db38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd2ac50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2a0d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd31278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2a3eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd35828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2a80f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd39e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2aa208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd3e400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2ae400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd3fa58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2aed68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd45080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2b16a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcc65f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2b1f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcc8c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2b49b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dccd278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2b8358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcce828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2b8cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcd1e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2bb630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcd4518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2bbef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcd8320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c24fe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcd90f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2513c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcd9eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c251978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcdbc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c251f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcdeac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c255518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcdf908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c255ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dce2748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2580f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dce7e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2586a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcf05c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c25ae48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcf5c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c261780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcfe400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c264ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dd04be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c26a710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc8d2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c26cf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc939e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2747b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc9c160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c27a0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc9fd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1fd9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dca5978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2ee358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcac550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2e4400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcb3128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c23eac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcb6cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c246ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcbc8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c202d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dcc44a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c207048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc4c0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c209320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc4cbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20d6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc4e748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20db38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc4f2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20f048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc4fdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20f518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc51978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20f978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc534e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c20fda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc53fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c212208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc56b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2126d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc5a7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c212b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc5d438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c230668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc630b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c233400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc64d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2361d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc699e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c236ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc6d5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c239c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc712e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c23ca58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc73ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1c0828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc77ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1c1588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc7cac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1c6320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc809b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1ca0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc092b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1cada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc0ea58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1ce7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc182e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1d4240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc1da90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1d7be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc272b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1dd5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc2da90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1e0fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc36278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c220400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc3ca58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2260b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc412b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2297f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc44be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2220b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbcb588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1ed3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbceef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1ee710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbd5780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1f3a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbdc0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1f5d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbdf9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1fc128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbe2d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c17e470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbe60f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c17ecc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbe8438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1835c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbea828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c183e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbec9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c185748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbeebe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c185fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbf1d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c188898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbf2eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c19b940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbf70b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c19e208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbf9278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c19e940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dc00390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c193208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db87908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1a1da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db8cef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1a87b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db95470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1ad160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db9c9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1b1b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dba5080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1b6470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbab710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1b9d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbafd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c13f6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbb8208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1427f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbbc240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c148048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbbea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c148e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbc3390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c14ba20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dbc4c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c14d6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db48518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2e8240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db4ad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2ea278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db505f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2ee7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db52eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c215cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db556a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c21b240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db59b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c18b978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db5c160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c18bdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db5f630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c198358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db60b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2996d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db65048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1509b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db67518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c194278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db699b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c194c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db6bef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c18d780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db6feb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c156198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db73e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c156f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db77e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c159e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db7be48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c165400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db7fe10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1680f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db83dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c168da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db08cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c16ba90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db0ccf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c16e940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db10b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1717f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db15160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c174550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db17400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c174fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db181d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c176860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db1db00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1795f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db217f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c179f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db26518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0febe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db2a208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c100828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db2be80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c103470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db30ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c108080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db358d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c108c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db3a0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2dc9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db3c908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c163a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db40198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c163438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db41940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c109da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dac7160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c10c8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dac8940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1113c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dace208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c111e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dacf9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c114978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dad1f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1164a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dad64a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c116e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dad7a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c11a828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dadc080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c11d1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dadd588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c11db70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dadfb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c11f588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dae4160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c11ff60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dae6780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c123940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daeaa20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c128d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daeecc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c12dbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daf2f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c131a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daf91d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c134908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dafd400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c136828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db00550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1398d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958db05358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0be080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da87860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c0550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da88d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c0f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da8e240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c38d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da8f6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c6240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da92b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c6ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da950b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c95c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da99630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0c9f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da9aba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0cd828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da9dcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0cf1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daa1eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c125b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daa6fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0d2940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daae128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0d9710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dab1240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0db630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dab6358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0de550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958daba470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e0470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dabd630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e2358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dabf9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e5240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958dac3da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e5ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da46198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e94a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da48518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0e9d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da4a8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0eb710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da4dc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0ee0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da4ffd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0ee940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da52390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0f12e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da54da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0f1ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da59eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c085cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da60128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c088dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da643c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c08cdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da67668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c08fe48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da6c978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c092ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da6fb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c094f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da73da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c097eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da7a0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c09bf60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da7d7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c09dfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da84358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0a4080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da089b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0a7320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da0f0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0aa4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da12710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0ad668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da16e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0b0898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da1d4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0b3a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da21b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0b6be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da23fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bdef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da28470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3c0518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da29908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0f3dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da2cd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0fb7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da31198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c07d0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da305f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c07dd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da339b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c080390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da37e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c080e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da3a2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c083a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da3bda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c1652b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958da436a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c03f710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1a7f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0426d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1ae4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0467f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b06a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c04a7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b4240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c04aef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b6748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c04b9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b6f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c438630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b9748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c42f3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1b9f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c42fc88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1bb748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c42f278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1bbf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c42fe80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1bc6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bd198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1bceb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bd240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1bf6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bd860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1c7198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c3bd470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1c8d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c055438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1ce128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0560f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1cfcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c056b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1d3978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0599b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1d9588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c05c828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1dd208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c05e6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1dee48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c061518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1e4a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0734a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1676a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0772e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c16c320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c079198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1704a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c07c048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c175518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7bb048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c179668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7bbf98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c17c6d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7bef60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c181710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7c1e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c185828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7c5e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c18a9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7c7e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c18cc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7cae10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c191080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7ce7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1934e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7d1208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c195898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7d1b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c197be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7d4518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c199fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7d4e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c19e390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0657b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c19f710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c068358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c126358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c0687f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c129a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c06de48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c12c208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c070400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c12e9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7d7eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c134128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7dca90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1358d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7de668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c13a0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e1160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c13b8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e1c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1400f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e47f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c140c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e7358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c142828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e7898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1443c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7e7d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c144f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7ea278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c146b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f94fc7ea710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c1476d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c128358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c14a240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c128c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f958c14add8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f952c2e8e48>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=74.89, Precision=78.79, FScore=76.79, CompleteMatch=9.01, TaggingAccuracy=91.17)'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-19-f034bd79213f>:1: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 197 variables.\n",
      "INFO:tensorflow:Converted 197 variables to const ops.\n"
     ]
    }
   ],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('export/model-tiny.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab-tiny.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
